﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace Hec.TextMining
{
    public class TextMining
    {
        public string Source { get; private set; }
        public IEnumerable<KeyValuePair<Document, string>> AllTextsRaw;
        public IEnumerable<KeyValuePair<Document, List<string>>> AllTexts;
        public IEnumerable<string> RawFrequency_VectorSpaceModel;
        public IEnumerable<KeyValuePair<Document, List<KeyValuePair<string, Cell>>>> VSM;

        public TextMining(string source)
        {
            Source = source;

            var directoryInfo = new DirectoryInfo(Source);

            AllTextsRaw = directoryInfo.GetFiles()
                            .Where(x => x.Extension != ".dropbox")
                            .Select(file =>
                            {
                                var text = File.ReadAllText(file.FullName);

                                return new KeyValuePair<Document, string>(new Document(file.FullName), text);
                            })
                            .ToList();
        }

        public TextMining(IList<KeyValuePair<Document, List<string>>> mockTexts)
        {
            AllTexts = mockTexts;
        }

        public TextMining(IList<KeyValuePair<Document, List<KeyValuePair<string, Cell>>>> fromSerialization)
        {
            VSM = fromSerialization;
        }

        public List<string> getDistinctWords()
        {
            var distinctWords = 
                    AllTextsRaw
                    //pour chaque texte, retire la liste de mots uniques triés en ordre alpha
                        .Select((keyValue, index) =>
                        {
                            var distinct = keyValue.Value
                                               .Split(' ')
                                               .Distinct()
                                               .OrderBy(y => y)
                                               .Select(value =>
                                               {
                                                   return value.ToString();
                                               })
                                               .ToList();
                            //                              distincts
                            //                      key     [mot1, mot2, ...]                                                    
                            return new KeyValuePair<Document, List<string>>(keyValue.Key, distinct);
                        })
                    //tous donc la liste de KeyValuePair<string, List<string>>
                        .SelectMany(x => x.Value.ToList())
                        .Distinct()
                        .OrderBy(z => z)
                        .Select(value => value.ToString())
                        .ToList();

            return distinctWords;
        }

        public List<string> getDistinctCharacters()
        {
            var distinctsAllText = AllTextsRaw.Select((keyValue, x) =>
                                            {
                                                var distinct = keyValue.Value
                                                                .ToCharArray()
                                                                .Distinct()
                                                                .OrderBy(y => y)
                                                                .Select(value =>
                                                                {
                                                                    return value.ToString();
                                                                }).ToList();
                                                return distinct;
                                            }).ToList()
                                            .SelectMany(x => x.ToList())
                                            .Distinct()
                                            .OrderBy(z => z)
                                            .Select(value => value.ToString())
                                            .ToList();

            return distinctsAllText;
        }

        public void _1_enleverPonctuation()
        {
            /*vecteur de charactères uniques obtenus avec la méthode getDistinctCharacters*/
            /*
                         
             
             ­        '  -  –  —        	  
  
  !  "  #  $  %  &  (  )  *  ,  .  /  :  ;  ?  @  [  ]  _  |  ~  ¦  ‘  ’  “  ”  ¢  €  +  <  =  >  »  §  ©  ®  ·  …  •  ‰  0  ½  1  2  3  4  5  6  7  8  9  a  á  ã  b  c  d  e  é  f  g  h  i  í  ï  j  k  l  m  n  o  ó  ö  p  q  r  s  š  t  ™  u  ü  v  w  x  y  z 
              
             
            ­   
             */

            AllTextsRaw = AllTextsRaw.Select((keyValue, x) =>
                            {
                                var value =  keyValue.Value
                                                //.Replace("'", " ")
                                                .Replace("-", " ")
                                                .Replace("–", " ")
                                                .Replace("—", " ")
                                                .Replace("!", " ")
                                                .Replace("\"", " ")
                                                .Replace("#", " ")
                                                .Replace("$", " ")
                                                .Replace("%", " ")
                                                .Replace("&", " ")
                                                .Replace("(", " ")
                                                .Replace(")", " ")
                                                .Replace("*", " ")
                                                .Replace(",", " ")
                                                .Replace(".", " ")
                                                .Replace("/", " ")
                                                .Replace(":", " ")
                                                .Replace(";", " ")
                                                .Replace("?", " ")
                                                .Replace("@", " ")
                                                .Replace("[", " ")
                                                .Replace("]", " ")
                                                .Replace("_", " ")
                                                .Replace("|", " ")
                                                .Replace("~", " ")
                                                .Replace("¦", " ")
                                                //.Replace("‘", "")
                                                //.Replace("“", "")
                                                //.Replace("”", "")
                                                .Replace("¢", " ")
                                                .Replace("€", " ")
                                                .Replace("+", " ")
                                                .Replace("<", " ")
                                                .Replace("=", " ")
                                                .Replace(">", " ")
                                                .Replace("»", " ")
                                                .Replace("§", " ")
                                                .Replace("©", " ")
                                                .Replace("®", " ")
                                                .Replace("·", " ")
                                                .Replace("…", " ")
                                                .Replace("•", " ")
                                                .Replace("‰", " ")
                                                .Replace("™", " ")
                                                .Replace("\t", " ")
                                                .Replace("\n", " ")
                                                .Replace("\r", " ");
                                return new KeyValuePair<Document,string>(new Document(keyValue.Key.FullName), value);
                            })
                            .ToList();
        }

        public void _2_enleverApostrophes()
        {
            AllTextsRaw = AllTextsRaw.Select((keyValue, x) =>
                                    {
                                        var value = keyValue.Value
                                                    .Replace("'", " ")
                                                    .Replace("‘", " ")
                                                    .Replace("’", " ")
                                                    .Replace("“", " ")
                                                    .Replace("”", " ");
                                        return new KeyValuePair<Document, string>(new Document(keyValue.Key.FullName), value);
                                    })
                                .ToList();
        }

        public void _3_enleverMotsCourts(int minSize)
        {
            AllTexts = AllTextsRaw.Select((keyValue, x) =>
                                        {
                                            var initial = keyValue.Value
                                                            .Split(' ')
                                                            .ToList();

                                            var final = initial
                                                            .Where(y => y.Length > minSize)
                                                            .ToList();
                                            return new KeyValuePair<Document, List<string>>(keyValue.Key, final);
                                        })
                                .ToList();
        }

        public void _4_tronquerMotsLongs(int maxLength)
        {
            AllTexts = AllTexts
                        .Select((keyValue, y) =>
                                {
                                    var final = keyValue.Value
                                                .Select(x =>
                                                {
                                                    var troncated = string.Empty;
                                                    if (x.Length > maxLength)
                                                    {
                                                        troncated = x.Substring(0, maxLength);
                                                    }
                                                    else
                                                    {
                                                        troncated = x;
                                                    }
                                                    
                                                    return troncated;
                                                })
                                                .ToList();

                                    //var result = string.Join(" ", final.ToArray());
                                    return new KeyValuePair<Document, List<string>>(keyValue.Key,  final);
                                })
                        .ToList();
        }

        public void _5_rawFrequency_VectorSpaceModel(int takeAmount)
        {
            var many = AllTexts.Select(x =>
                            {
                                var list = x.Value.Select(y =>
                                {
                                    var transform = new KeyValuePair<Document, string>(x.Key, y);
                                    return transform;
                                }).ToList();
                                return list;
                            })
                            .SelectMany(x => x)
                            .ToList();

            // la liste des documents distincts
            var rows = AllTexts.Select(x => new Document(){FullName = x.Key.FullName, TotalTermCount = x.Value.Count()}).Distinct().ToList();
            // la liste des terms distincs dans tous les documents. 
            var columns = many.Select(x => x.Value).Distinct().ToList();

            //for test purpose
            if (takeAmount > -1) rows = rows.Take(takeAmount).ToList();

            var vsm =
                        rows
                        .AsParallel()
                        .Select((row, index) =>
                        {
                            row.ID = index;
                            var array = columns.Select(column =>
                            {
                                var one = many
                                            .Where(x => x.Key.Equals(row)).ToList();
                                var two = one
                                            .Where(x => x.Value == column).ToList();
                                var count = two
                                            .Count();

                                var keyValue = new KeyValuePair<string, Cell>(column.SanitizeXmlString(), new Cell(count, row));
                                return keyValue;

                            }).ToList();

                            var result = new KeyValuePair<Document, List<KeyValuePair<string, Cell>>>(row, array);
                            return result;
                        })
                            .ToList();
            VSM = vsm;
        }

        public void _6_discriminant(double minPourcentage, double maxPourcentage)
        {
            var totalDocs = (double)VSM.Count();

            //1st step : get words to keep
            var result =
                        VSM.SelectMany(x => x.Value)
                            .GroupBy(x => x.Key)
                            .Select(x =>
                                new
                                {
                                    Key = x.Key,
                                    Count = x.ToList().Where(y => y.Value.Frequency > 0).Count(),
                                    Ratio = (double)x.ToList().Where(y => y.Value.Frequency > 0).Count() / totalDocs
                                    //Count = x.ToList().Sum(y => y.Value.Frequency ), 
                                    //Ratio = x.ToList().Sum(y =>  y.Value.Frequency)
                                })
                            .Where(x => x.Ratio > minPourcentage && x.Ratio < maxPourcentage)
                            .ToList();

            VSM = VSM
                    .AsParallel()
                    .Select(document => 
                    {
                        var vector = document.Value
                                        .Where(x => result.Select(z => z.Key).Contains(x.Key))
                                        .ToList();
                        document.Value = vector;

                        return document; 
                    })
                    .ToList();

        } 

        public void _7_calculer_poids_tf_idf()
        {
            //TFij = Fij/Fj
            //      TFij is the relative frequency of the term i in the document j
            //      IDFi is the logarithm of the inverse of the proportion of documents where the term i appears

            //IDFi = log(N/ni)

            double N = VSM.Count();

            VSM.Select(document =>
                {
                    var result = document.Value.AsParallel().Select(term =>
                        {
                            #region calculs le poids tf_idf
                            double ni = VSM.SelectMany(x => x.Value).Where(x => x.Key == term.Key && x.Value.PresencAbsence).Count();
                            var idfi = ni > 0 ? Math.Log(N / ni, 10) : 0.0;

                            var tfij = term.Value.Frequency / document.Key.TotalTermCount;

                            term.Value.TF_IDF = tfij * idfi; 
                            #endregion

                            return term;
                        }).ToList();

                    document.Value = result;
                    return document;
                }).ToList();
        }

        public void _7_1_calculer_distances()
        {
            VSM = 
                    VSM
                        .AsParallel()
                        .Select(document => 
                    {
                        var furterDocuments = VSM
                                                .Where(furtherDocuments => furtherDocuments.Key.ID > document.Key.ID)
                                                .Select(furtherDoc =>
                                                {
                                                    //pour chaque terme du document courant
                                                    var toSum = document.Value.Select(z =>
                                                                    {
                                                                        var term =
                                                                                    furtherDoc
                                                                                        .Value
                                                                                        .Single(x => x.Key == z.Key);
                                                                        return new 
                                                                        { 
                                                                            delta_carre_tf_idf = Math.Pow(term.Value.TF_IDF - z.Value.TF_IDF, 2) 
                                                                        };
                                                                    });

                                                    var sum = toSum.Sum(x => x.delta_carre_tf_idf);
                                                    return new {Document = furtherDoc.Key, euclidian_on_TF_IDF= Math.Sqrt(sum)};
                                                })
                                                .ToList();
                        document.Key.Distances = furterDocuments
                                                    .Select(x => new Hec.TextMining.KeyValuePair<Document, Distance>(x.Document,  new Distance() { Euclidian_TF_IDF = x.euclidian_on_TF_IDF }))
                                                    .ToList();
                
                        return document;
                    })
                    .ToList();

        }
    }
}
